from pyspark import SparkConf
from pyspark import SparkContext

if __name__ == '__main__':
    conf = SparkConf().setMaster("local[6]").setAppName("spark01")
    sc = SparkContext(conf=conf)

    # 需求： 取出每个分区相同的key相应值的最大值，然后相加

    # 数据:
    lst1 = [("a", 3), ("a", 2), ("c", 4), ('b', 7), ("a", 92), ("c", 42)]
    rdd1 = sc.parallelize(lst1, 4)

    print(rdd1.getNumPartitions())
    # rdd2 = rdd1.coalesce(3, True) # 重新(缩减)分区，第二个参数:shuffle:Boolean
    rdd2 = rdd1.repartition(3)  # 重新分区,本质上调用的就是coalesce(shuffle=True)
    print(rdd2.getNumPartitions())
